import json
import os
import platform
import re
import sys
import time
import traceback

import facade
import requests
from vip.DiscernCode import DiscernCode
from xjlibrary.mprocesspoll.process_distributed.BaseTask import BaseTask
from xjlibrary.network.MyIP import GetLocalIPByPrefix, get_local_ip
from xjlibrary.our_file_dir import BaseDir
from xjlibrary.tools.BaseUrl import BaseUrl

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
dirPath = BaseDir.get_new_path(TopPath, "download", "EI", "download", "excel")



class NodeTask(BaseTask):
    def __init__(self):
        super().__init__()
        # 连接到服务器，也就是运行task_master.py的机器:
        sysstr = platform.system()
        if sysstr == "Windows":
            self.ip = GetLocalIPByPrefix("192.168.")
        else:
            self.ip = get_local_ip("enp2s0")
        self.server_addr = self.ip
        self.port = 5000
        self.init_node(self.server_addr, self.port)


class DownIndexByAN(object):
    def __init__(self):
        self.configfile = BaseDir.get_new_path(curPath, "db.ini")
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(self.configfile, "db", logger=self.logger)
        # 8207 8082 8012
        self.Proxies = {
            "http": "192.168.30.36:8182",
            "https": "192.168.30.36:8182"  # key是指目标网站的协议
        }
        self.year_floor = None
        self.year_ceil = None
        self.ac_list = []
        self.NumPerPage = 100  # 检索结果页每页25条
        self.NumAllPage = 5200  # 最大条数限制
        self.sn = requests.Session()
        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
        self.sn.headers['User-Agent'] = self.UserAgent
        self.init_form()
        self.discern_code_time = int(time.time()) - 300
        self.SEARCHID = ""
        self.sessionId = ""
        self.node = NodeTask()
        self.init_node()

    def init_node(self):
        self.node.conn()
        self.node.set_task_result_obj()

    def init_form(self):
        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
        self.t = int(round(time.time() * 1000))
        self.SearchChinaData = {
            "usageOrigin": "searchform",
            "usageZone": "quicksearch",
            "editSearch": "",
            "isFullJsonResult": "true",
            "angularReq": "true",
            "CID": "searchSubmit",
            "searchtype": "Quick",
            "origin": "searchform",
            "category": "quicksearch",
            "section1": "NO-LIMIT",
            "searchWord1": "China",
            # "allDb": "1",
            "database": "1",
            "yearselect": "yearrange",
            "startYear": "1884",
            "endYear": "2020",
            "updatesNo": "1",
            "language": "NO-LIMIT",
            "doctype": "NO-LIMIT",
            "sort": "relevance",
            "treatmentType": "NO-LIMIT",
            "searchStartTimestamp": str(int(round(time.time() * 1000))),
            "_": str(self.t)
        }

    def init_search_id_old(self):
        """
        重庆大学专用
        :return:
        """

        def init_home():
            """
            选择学校
            :return:
            """
            url = r'https://www.engineeringvillage.com/home.url'
            header = {
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                "Accept-Encoding": "gzip, deflate",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                "Host": "www.engineeringvillage.com",
                "Upgrade-Insecure-Requests": "1",
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
            }
            BoolResult, errString, r = facade.BaseRequest(url,
                                                          sn=self.sn,
                                                          headers=header,
                                                          allow_redirects=True,
                                                          verify=False,
                                                          proxies=self.Proxies,
                                                          timeout=60)
            self.logger.info(errString)
            if not BoolResult:
                return False
            self.logger.info("首页访问完成")
            print(r.url)
            BaseDir.single_write_file(r.text, "./test.html")
            # 重庆大学独有
            if r.text.find("Chongqing University Library, General Access") > -1:
                obj = re.search(r'csrfToken\.value = "(.*?)"', r.text)
                csrfToken = obj.group(1)
                print(csrfToken)
                url = "https://www.engineeringvillage.com/customer/authenticate.url"
                postdate = {
                    "path_choice": "13022631",
                    # "remember_path_flag": "true",
                    "origin": "pathChoice",
                    "zone": "main",
                    "auth_type": "CANDIDATE_PATH",
                    "csrfToken": csrfToken
                }
                print(postdate)
                headers = {
                    "Content-Type": "application/x-www-form-urlencoded",
                    "Origin": "https://www.engineeringvillage.com",
                    "Referer": "https://www.engineeringvillage.com/search/quick.url",
                    "Upgrade-Insecure-Requests": "1",
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"
                }
                BoolResult, errString, r = facade.BaseRequestPost(url=url,
                                                                  sn=self.sn,
                                                                  data=postdate,
                                                                  headers=headers,
                                                                  proxies=self.Proxies,
                                                                  allow_redirects=True,
                                                                  endstring="",
                                                                  verify=False,
                                                                  timeout=60)
                if BoolResult:
                    print("suceess")
                    print(r.url)
                else:
                    print("failed")
                    sys.exit(-1)

        BaseSearchUrl = "https://www.engineeringvillage.com/search/quick.url"
        headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Host": "www.engineeringvillage.com",
            "Pragma": "no-cache",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"
        }
        bool, strins, r = facade.BaseRequest(BaseSearchUrl,
                                             sn=self.sn,
                                             mark="search-form-top",
                                             headers=headers,
                                             proxies=self.Proxies,
                                             verify=False,
                                             timeout=(30, 60))
        if bool:
            """
            如果有 代表失败
            """
            print("quick 失败")
            init_home()
            return False

        url = "https://www.engineeringvillage.com/rest/sessionsettings/0?overrideMethod=PATCH"
        postdate = {
            "selectedMask": "1"
        }
        header = {"Accept": "application/json, text/javascript, */*; q=0.01",
                  "Accept-Encoding": "gzip, deflate, br",
                  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                  "Connection": "keep-alive",
                  "Content-Type": "application/json",
                  "Host": "www.engineeringvillage.com",
                  "Origin": "https://www.engineeringvillage.com",
                  "Referer": "https://www.engineeringvillage.com/search/quick.url",
                  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
                  "X-HTTP-Method-Override": "PATCH",
                  "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
                  "X-Requested-With": "XMLHttpRequest"
                  }
        BoolResult, errString, r = facade.BaseRequestPost(url,
                                                          sn=self.sn,
                                                          endstring="",
                                                          mark="advOptionsDisplay",
                                                          headers=header,
                                                          data=json.dumps(postdate),
                                                          verify=False,
                                                          timeout=30
                                                          )
        if not BoolResult:
            print(r.text)
            print(r.url)
        parastring = ""
        for key, value in self.SearchChinaData.items():
            parastring += key + "=" + value + "&"
        parastring = parastring[:-1]

        searchUrl = "https://www.engineeringvillage.com/search/submit.url" + "?" + parastring
        bool, strins, r = facade.BaseRequest(searchUrl,
                                             sn=self.sn,
                                             proxies=self.Proxies,
                                             headers=header,
                                             mark="results",
                                             endstring="",
                                             allow_redirects=True,
                                             verify=False,
                                             timeout=(30, 60))
        BaseDir.single_write_file(r.text, "./err1.html")
        print(r.url)
        if not bool:
            sys.exit("搜索请求失败")
        try:
            sessionId = json.loads(r.text)["searchMetaData"]["searchesEntity"]["sessionId"]
        except:
            print("解析出错")
            sys.exit(-1)
        dic = BaseUrl.urlQuery2Dict(r.url)
        SEARCHID = dic["SEARCHID"]
        self.SEARCHID = SEARCHID
        self.sessionId = sessionId
        print(self.SEARCHID)
        print(self.sessionId)
        print("请求成功")

    def init_search_id(self):
        url = r'https://www.engineeringvillage.com/home.url'
        header = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Host": "www.engineeringvillage.com",
            "Upgrade-Insecure-Requests": "1",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "none",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }
        BoolResult, errString, r = facade.BaseRequest(url,
                                                      sn=self.sn,
                                                      headers=header,
                                                      allow_redirects=True,
                                                      verify=False,
                                                      proxies=self.Proxies,
                                                      timeout=60)
        self.logger.info(errString)
        if not BoolResult:
            return False

        url = "https://www.engineeringvillage.com/rest/sessionsettings/0?overrideMethod=PATCH"
        postdate = {
            "selectedMask": "1"
        }
        header = {"Accept": "application/json, text/javascript, */*; q=0.01",
                  "Accept-Encoding": "gzip, deflate",
                  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                  "Content-Type": "application/json",
                  "Host": "www.engineeringvillage.com",
                  "Origin": "https://www.engineeringvillage.com",
                  "Referer": "https://www.engineeringvillage.com/search/quick.url",
                  "Sec-Fetch-Mode": "cors",
                  "Sec-Fetch-Site": "same-origin",
                  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
                  "X-HTTP-Method-Override": "PATCH",
                  "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
                  "X-Requested-With": "XMLHttpRequest"
                  }
        BoolResult, errString, r = facade.BaseRequestPost(url,
                                                          sn=self.sn,
                                                          endstring="",
                                                          mark="advOptionsDisplay",
                                                          headers=header,
                                                          data=json.dumps(postdate),
                                                          verify=False,
                                                          timeout=30
                                                          )
        if not BoolResult:
            print(r.text)
            print(r.url)
            return False

        parastring = ""
        for key, value in self.SearchChinaData.items():
            parastring += key + "=" + value + "&"
        parastring = parastring[:-1]

        searchUrl = "https://www.engineeringvillage.com/search/submit.url" + "?" + parastring
        bool, strins, r = facade.BaseRequest(searchUrl,
                                             sn=self.sn,
                                             proxies=self.Proxies,
                                             headers=header,
                                             mark="results",
                                             endstring="",
                                             allow_redirects=True,
                                             verify=False,
                                             timeout=(30, 60))
        BaseDir.single_write_file(r.text, "./err1.html")
        print(r.url)
        if not bool:
            sys.exit("搜索请求失败")
        try:
            sessionId = json.loads(r.text)["searchMetaData"]["searchesEntity"]["sessionId"]
        except:
            print("解析出错")
            sys.exit(-1)
        dic = BaseUrl.urlQuery2Dict(r.url)
        SEARCHID = dic["SEARCHID"]
        self.SEARCHID = SEARCHID
        self.sessionId = sessionId
        print(self.SEARCHID)
        print(self.sessionId)
        print("请求成功")
        return True

    # 插入数据库
    def SelectFromdb(self):
        # 数据库连接
        sSql = "select `AccessionNumber`,`docid` from `articlenew` where `stat`=0 LIMIT 500"
        rows = self.mysqlutils.SelectFromDB(sSql)
        if rows is None:
            print("select出现错误 请检查")
            sys.exit(-1)
        return rows

    def discern_code(self, jpgpath):
        """
        识别验证码通过云打码
        :return:
        """
        if int(time.time()) - self.discern_code_time > 60 * 5:  #
            boolresult, code = DiscernCode(self.logger, codetype=3006).discern_code(jpgpath)
            BaseDir.single_add_file("./code.txt", "***code is :{}\n".format(code))
            self.discern_code_time = int(time.time())
            if boolresult:
                return True, code
            else:
                return False, ""
        else:
            self.logger.info("时间还没到休息60秒 差时为{} 应该相隔5分钟".format(int(time.time()) - self.discern_code_time))
            time.sleep(60)
            return False, ""

    def commitlistcpx(self, paradic):
        # 选择需要下载的url
        urlselect = "https://www.engineeringvillage.com/basket/mark.url"
        header = {"Accept": "*/*",
                  "Accept-Encoding": "gzip, deflate",
                  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                  "Cache-Control": "no-cache",
                  "Pragma": "no-cache",
                  "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
                  "Host": "www.engineeringvillage.com",
                  "Origin": "https://www.engineeringvillage.com",
                  "Referer": "https://www.engineeringvillage.com/search/quick.url?SEARCHID={}&COUNT=1&usageOrigin=header&usageZone=evlogo".format(
                      self.SEARCHID),
                  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
                  "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
                  "X-Requested-With": "XMLHttpRequest"
                  }

        handle = []
        docid = []
        for key, value in paradic.items():
            handle.append(str(key))
            docid.append(value)

        postData = {
            "handle": handle,
            "docid": docid,
            "searchid": self.SEARCHID
        }
        print(postData)
        bool, strins, r = facade.BaseRequestPost(urlselect,
                                                 sn=self.sn,
                                                 data=postData,
                                                 mark="status",
                                                 headers=header,
                                                 endstring="",
                                                 proxies=self.Proxies,
                                                 verify=False,
                                                 timeout=(30, 60))
        print(r.text)
        if not bool:
            sys.exit("提交数据请求失败")

    def getcsrfToken(self):
        url = "https://www.engineeringvillage.com/rest/sessionsettings/0?overrideMethod=PATCH"
        header = {"Accept": "application/json, text/javascript, */*; q=0.01",
                  "Accept-Encoding": "gzip, deflate",
                  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                  "Content-Type": "application/json",
                  "Host": "www.engineeringvillage.com",
                  "Origin": "https://www.engineeringvillage.com",
                  "Sec-Fetch-Mode": "cors",
                  "Sec-Fetch-Site": "same-origin",
                  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
                  "X-HTTP-Method-Override": "PATCH",
                  "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
                  "X-Requested-With": "XMLHttpRequest"
                  }

        payload = {"dlRemoveSelected": "false", "dlSummary": "false", "dlFormat": "excel", "dlOutput": "detailed"}
        Feature = "csrfToken"
        bool, strins, r = facade.BaseRequestPost(url,
                                                 sn=self.sn,
                                                 data=json.dumps(payload),
                                                 mark=Feature,
                                                 headers=header,
                                                 endstring="",
                                                 proxies=self.Proxies,
                                                 verify=False,
                                                 timeout=(30, 60))

        if not bool:
            sys.exit("csrfToken请求失败")
        csrfToken = json.loads(r.text)["csrfToken"]
        return csrfToken

    def writeexcel(self, csrfToken, num):
        submitdata = {
            "sessionid": self.sessionId,
            "docidlist": "",
            "folderid": "",
            "database": "1",
            "baseaddress": "www.engineeringvillage.com",
            "sortBy": "",
            "sortDirn": "",
            "outputLocation": "mypc",
            "downloadformat": "excel",
            "csrfToken": csrfToken,
            "displayformat": "detailed",
            "filenameprefix": "Engineering_village",
            "usageOrigin": "searchresults"
        }

        url = "https://www.engineeringvillage.com/delivery/download/submit.url"
        header = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
                  "Accept-Encoding": "gzip, deflate",
                  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                  "Content-Type": "application/x-www-form-urlencoded",
                  "Host": "www.engineeringvillage.com",
                  "Sec-Fetch-Mode": "navigate",
                  "Sec-Fetch-User": "?1",
                  "Sec-Fetch-Site": "same-origin",
                  "Origin": "https://www.engineeringvillage.com",
                  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
                  "Upgrade-Insecure-Requests": "1",
                  }
        r = self.sn.post(url,
                         proxies=self.Proxies,
                         data=submitdata,
                         headers=header,
                         verify=False,
                         timeout=(30, 60))
        if not os.path.exists(dirPath):
            os.makedirs(dirPath)
        filepath = os.path.join(dirPath, str(int(time.time())) + "_" + str(num) + ".xlsx")
        with open(filepath, 'wb') as f:
            f.write(r.content)

    # 插入数据库
    def Updatelist(self, list):
        strins = tuple(list)
        sSql = "update `articlenew` set `stat`=2 where AccessionNumber in {}".format(strins)
        # 数据库连接
        result = self.mysqlutils.ExeSqlToDB(sSql)
        if not result:
            print("更新语句执行错误")
            sys.exit(-1)


if __name__ == "__main__":
    down = DownIndexByAN()
    while True:
        try:
            down.sn = requests.Session()
            down.sn.headers['User-Agent'] = down.UserAgent
            if down.node.task.empty():
                print("time.sleep(10)")
                time.sleep(10)
                continue
            if down.init_search_id():
                rows = []
                try:
                    for _ in range(500):
                        ids = down.node.task.get(timeout=10)
                        rows.append(ids)
                except:
                    pass
                num = len(rows)
                if len(rows) < 500:
                    bool = False
                if num == 0:
                    continue
                index = 1
                paradic = {}
                idlist = []
                for docid in rows:
                    paradic[index] = docid[1]
                    idlist.append(docid[0])
                    index += 1
                down.commitlistcpx(paradic)
                csrfToken = down.getcsrfToken()
                down.writeexcel(csrfToken, num)
                down.Updatelist(idlist)
            time.sleep(5)
        except:
            traceback.print_exc()
